Amazon CloudWatch Observability アドオンを利用して SageMaker HyperPod を監視してみた

###################################################
# IAM Role for SageMaker HyperPod Cluster
###################################################
resource "aws_iam_role" "hyperpod" {
  name = "${local.prefix}-hyperpod-role"
  assume_role_policy = jsonencode({
    Version = "2012-10-17",
    Statement = [
      {
        Effect = "Allow",
        Principal = {
          Service = "sagemaker.amazonaws.com"
        },
        Action = "sts:AssumeRole"
      }
    ]
  })
}

resource "aws_iam_role_policy_attachment" "hyperpod_managed" {
  role       = aws_iam_role.hyperpod.name
  policy_arn = "arn:aws:iam::aws:policy/AmazonSageMakerClusterInstanceRolePolicy"
}

resource "aws_iam_role_policy_attachment" "hyperpod_observability" {
  role       = aws_iam_role.hyperpod.name
  policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
}

data "aws_iam_policy_document" "hyperpod_vpc" {
  statement {
    effect = "Allow"
    actions = [
      "ec2:AssignPrivateIpAddresses",
      "ec2:CreateNetworkInterface",
      "ec2:CreateNetworkInterfacePermission",
      "ec2:DeleteNetworkInterface",
      "ec2:DeleteNetworkInterfacePermission",
      "ec2:DescribeNetworkInterfaces",
      "ec2:DescribeVpcs",
      "ec2:DescribeDhcpOptions",
      "ec2:DescribeSubnets",
      "ec2:DescribeSecurityGroups",
      "ec2:DetachNetworkInterface",
      "ec2:ModifyNetworkInterfaceAttribute",
      "ec2:UnassignPrivateIpAddresses",
      "ecr:BatchGetImage",
      "ecr:GetAuthorizationToken",
      "ecr:GetDownloadUrlForLayer",
      "eks-auth:AssumeRoleForPodIdentity"
    ]
    resources = ["*"]
  }

  statement {
    effect = "Allow"
    actions = [
      "ec2:CreateTags",
    ]
    resources = ["arn:aws:ec2:*:*:network-interface/*"]
  }
}

resource "aws_iam_policy" "hyperpod_vpc" {
  name        = "${local.prefix}-vpc-policy"
  description = "IAM policy for SageMaker HyperPod VPC"
  policy      = data.aws_iam_policy_document.hyperpod_vpc.json
}

aws eks create-addon --cluster-name cluster-name
--addon-name amazon-cloudwatch-observability
--configuration-values "configuration json"

{
	"agent": {
		"config": {
			"logs": {
				"metrics_collected": {
					"kubernetes": {
						"kueue_container_insights": true,
						"enhanced_container_insights": true
					},
					"application_signals": {}
				}
			},
			"traces": {
				"traces_collected": {
					"application_signals": {}
				}
			}
		}
	}
}

resource "aws_eks_addon" "amazon_cloudwatch_observability" {
  cluster_name                = aws_eks_cluster.this.name
  addon_name                  = "amazon-cloudwatch-observability"
  resolve_conflicts_on_create = "OVERWRITE"
  configuration_values = jsonencode({
    agent = {
      config = {
        logs = {
          metrics_collected = {
            kubernetes = {
              kueue_container_insights    = true
              enhanced_container_insights = true
            }
            application_signals = {}
          }
        }
        traces = {
          traces_collected = {
            application_signals = {}
          }
        }
      }
    }
  })

  depends_on = [
    awscc_sagemaker_cluster.this,
  ]
}

apiVersion: apps/v1
kind: Deployment
metadata:
  name: cpu-memory-stress-deployment
  labels:
    app: cpu-memory-stress
spec:
  replicas: 2 # Podのレプリカ数を1に設定
  selector:
    matchLabels:
      app: cpu-memory-stress
  template:
    metadata:
      labels:
        app: cpu-memory-stress
    spec:
      containers:
        - name: cpu-memory-stress-container
          image: polinux/stress # stressツールを含む軽量なイメージ
          command: ['stress'] # 実行するコマンドを明示的に指定
          args:
            - '--cpu'
            - '4' # 4つのスレッドでCPUを負荷
            - '--vm'
            - '7' # 7つのスレッドでメモリを負荷
            - '--vm-bytes'
            - '4G' # 各スレッドが4 GiBのメモリを確保（合計28 GiB）
            - '--timeout'
            - '600s' # 600秒間（10分間）負荷をかける
          resources:
            requests:
              cpu: '4' # 最低限必要なCPUリソース（4 vCPU）
              memory: '28Gi' # メモリリクエスト
            limits:
              cpu: '4' # 最大4 vCPUを使用
              memory: '28Gi' # メモリ制限